{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem 1\n",
    "Load Iris data. Use data transformations a classification algorithm of your choice to achieve greater than 98% accuracy in cross-validation (cv = 3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data \n",
    "y = iris.target\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9791666666666666, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV] ....... n_neighbors=9, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9607843137254902, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] ...... n_neighbors=12, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=10, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=14, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9411764705882353, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9411764705882353, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=15, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ....... n_neighbors=11, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9411764705882353, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9411764705882353, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] ...... n_neighbors=13, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=15, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=11, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9607843137254902, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9791666666666666, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9803921568627451, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9803921568627451, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  16 out of  48 | elapsed:    0.1s remaining:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise',\n",
       "       estimator=KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=5, p=2,\n",
       "           weights='uniform'),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid=[{'weights': ['uniform', 'distance'], 'n_neighbors': [8, 9, 10, 11, 12, 13, 14, 15]}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [{'weights': [\"uniform\", \"distance\"], 'n_neighbors': [8,9,10,11,12,13,14,15]}]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "grid_search = GridSearchCV(knn_clf, param_grid, cv=3, verbose=3, n_jobs=-1,scoring=\"accuracy\")\n",
    "grid_search.fit(X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] ........ n_neighbors=8, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] ....... n_neighbors=8, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] ........ n_neighbors=9, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] ....... n_neighbors=9, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] ....... n_neighbors=10, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] ...... n_neighbors=10, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=12, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] ...... n_neighbors=14, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.8787878787878788, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=13, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=11, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] ....... n_neighbors=15, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9090909090909091, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.8787878787878788, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.8787878787878788, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] ...... n_neighbors=13, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] ...... n_neighbors=11, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.8181818181818182, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.8787878787878788, total=   0.0s\n",
      "[CV] ...... n_neighbors=15, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=12, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ....... n_neighbors=14, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9090909090909091, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.8787878787878788, total=   0.0s\n",
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  16 out of  48 | elapsed:    0.1s remaining:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ........ n_neighbors=8, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ........ n_neighbors=8, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ....... n_neighbors=8, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ....... n_neighbors=8, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] ........ n_neighbors=9, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] ....... n_neighbors=9, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] ........ n_neighbors=9, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ....... n_neighbors=9, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ....... n_neighbors=10, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ....... n_neighbors=10, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=13, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=10, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ...... n_neighbors=10, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ...... n_neighbors=13, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV] ....... n_neighbors=11, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=13, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=11, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] ....... n_neighbors=14, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=11, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ...... n_neighbors=11, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] ...... n_neighbors=14, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=12, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] ....... n_neighbors=15, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] ...... n_neighbors=12, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] ...... n_neighbors=12, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV] ...... n_neighbors=15, weights=distance, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] ...... n_neighbors=15, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "Fitting 3 folds for each of 16 candidates, totalling 48 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  16 out of  48 | elapsed:    0.0s remaining:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=uniform ..................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=8, weights=distance .................................\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] ........ n_neighbors=8, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV]  n_neighbors=8, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=uniform ..................................\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV]  n_neighbors=8, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=9, weights=distance .................................\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV]  n_neighbors=9, weights=uniform, score=0.9393939393939394, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=uniform .................................\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV]  n_neighbors=9, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] ....... n_neighbors=10, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=10, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.9393939393939394, total=   0.0s\n",
      "[CV] n_neighbors=10, weights=distance ................................\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=10, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=uniform .................................\n",
      "[CV] n_neighbors=14, weights=uniform .................................\n",
      "[CV]  n_neighbors=11, weights=uniform, score=0.8787878787878788, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=uniform, score=0.8787878787878788, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] ...... n_neighbors=14, weights=distance, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=11, weights=distance ................................\n",
      "[CV] n_neighbors=14, weights=distance ................................\n",
      "[CV]  n_neighbors=11, weights=distance, score=0.8787878787878788, total=   0.0s\n",
      "[CV]  n_neighbors=14, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=12, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=uniform .................................\n",
      "[CV] n_neighbors=12, weights=uniform .................................\n",
      "[CV]  n_neighbors=15, weights=uniform, score=0.9090909090909091, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.9696969696969697, total=   0.0s\n",
      "[CV] n_neighbors=12, weights=distance ................................\n",
      "[CV] n_neighbors=15, weights=distance ................................\n",
      "[CV]  n_neighbors=15, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV]  n_neighbors=12, weights=distance, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.9722222222222222, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV] ....... n_neighbors=13, weights=uniform, score=1.0, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=uniform .................................\n",
      "[CV]  n_neighbors=13, weights=uniform, score=0.8484848484848485, total=   0.0s\n",
      "[CV] n_neighbors=13, weights=distance ................................\n",
      "[CV]  n_neighbors=13, weights=distance, score=0.9722222222222222, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  16 out of  48 | elapsed:    0.0s remaining:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  48 out of  48 | elapsed:    0.1s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.9869281045751634"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "np.mean(cross_val_score(grid_search, X, y, cv=3, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The accuracy in cross-validation (cv=3) is 98.6928%**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem 2 \n",
    "Load MNIST data set. Use data transformations a classification algorithm(s) of your choice to achieve greater than 95% accuracy in cross-validation (cv = 3). There are many ways to achieve, so there are many correct answers. Use only algorithms that we studied. KNN is forbidden! Hint: Data is big, look for more efficient algorithms. Start with simple models, make it complex if needed. On you computer the code should run less than 30 minutes. I have a fast computer, it should run less 30 minutes on my as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "mnist = fetch_mldata('MNIST original')\n",
    "X = mnist.data\n",
    "y = mnist.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype uint8 was converted to float64 by StandardScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, error_score='raise',\n",
       "          estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=42, verbose=0, warm_start=False),\n",
       "          fit_params=None, iid=True, n_iter=10, n_jobs=1,\n",
       "          param_distributions={'n_estimators': <scipy.stats._distn_infrastructure.rv_frozen object at 0x1a1144dcc0>},\n",
       "          pre_dispatch='2*n_jobs', random_state=42, refit=True,\n",
       "          return_train_score='warn', scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from scipy.stats import randint\n",
    "\n",
    "param_distribs = {\n",
    "        'n_estimators': randint(low=10, high=30),\n",
    "        #'max_features': randint(low=15, high=20),\n",
    "    }\n",
    "\n",
    "forest_reg = RandomForestClassifier(random_state=42)\n",
    "# Set the number of iterations as 10\n",
    "rnd_search = RandomizedSearchCV(forest_reg, param_distributions=param_distribs,\n",
    "                                n_iter=10, n_jobs=1, cv=5, scoring='accuracy', random_state=42)\n",
    "rnd_search.fit(X_train_scaled, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9515429169578525"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.mean(cross_val_score(rnd_search, X_train_scaled, y_train, cv=3, scoring=\"accuracy\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Problem 3 \n",
    "Load CA housing data set. Use data transformations a regression algorithm(s) of your choice to achieve MSE less than $65,000 in predicting housing prices in cross-validation (cv = 3). There are many ways to achieve, so there are many correct answers. Use only algorithms that we studied. On you computer the code should run less than 30 minutes. I have a fast computer, it should run less 30 minutes on my as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longitude</th>\n",
       "      <th>latitude</th>\n",
       "      <th>housing_median_age</th>\n",
       "      <th>total_rooms</th>\n",
       "      <th>total_bedrooms</th>\n",
       "      <th>population</th>\n",
       "      <th>households</th>\n",
       "      <th>median_income</th>\n",
       "      <th>median_house_value</th>\n",
       "      <th>ocean_proximity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-122.23</td>\n",
       "      <td>37.88</td>\n",
       "      <td>41.0</td>\n",
       "      <td>880.0</td>\n",
       "      <td>129.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>8.3252</td>\n",
       "      <td>452600.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-122.22</td>\n",
       "      <td>37.86</td>\n",
       "      <td>21.0</td>\n",
       "      <td>7099.0</td>\n",
       "      <td>1106.0</td>\n",
       "      <td>2401.0</td>\n",
       "      <td>1138.0</td>\n",
       "      <td>8.3014</td>\n",
       "      <td>358500.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-122.24</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1467.0</td>\n",
       "      <td>190.0</td>\n",
       "      <td>496.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>7.2574</td>\n",
       "      <td>352100.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1274.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>558.0</td>\n",
       "      <td>219.0</td>\n",
       "      <td>5.6431</td>\n",
       "      <td>341300.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-122.25</td>\n",
       "      <td>37.85</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1627.0</td>\n",
       "      <td>280.0</td>\n",
       "      <td>565.0</td>\n",
       "      <td>259.0</td>\n",
       "      <td>3.8462</td>\n",
       "      <td>342200.0</td>\n",
       "      <td>NEAR BAY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   longitude  latitude  housing_median_age  total_rooms  total_bedrooms  \\\n",
       "0    -122.23     37.88                41.0        880.0           129.0   \n",
       "1    -122.22     37.86                21.0       7099.0          1106.0   \n",
       "2    -122.24     37.85                52.0       1467.0           190.0   \n",
       "3    -122.25     37.85                52.0       1274.0           235.0   \n",
       "4    -122.25     37.85                52.0       1627.0           280.0   \n",
       "\n",
       "   population  households  median_income  median_house_value ocean_proximity  \n",
       "0       322.0       126.0         8.3252            452600.0        NEAR BAY  \n",
       "1      2401.0      1138.0         8.3014            358500.0        NEAR BAY  \n",
       "2       496.0       177.0         7.2574            352100.0        NEAR BAY  \n",
       "3       558.0       219.0         5.6431            341300.0        NEAR BAY  \n",
       "4       565.0       259.0         3.8462            342200.0        NEAR BAY  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from six.moves import urllib\n",
    "import os\n",
    "DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml/master/\"\n",
    "HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n",
    "HOUSING_URL = DOWNLOAD_ROOT + \"datasets/housing/housing.tgz\"\n",
    "import pandas as pd\n",
    "housing = pd.read_csv('/Users/vannyzhang/Desktop/Machine Learning/Data/datasets/housing/housing.csv')\n",
    "housing.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)\n",
    "housing[\"income_cat\"] = np.ceil(housing[\"median_income\"] / 1.5)\n",
    "housing[\"income_cat\"].where(housing[\"income_cat\"] < 5, 5.0, inplace=True)\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "\n",
    "for train_index, test_index in split.split(housing, housing[\"income_cat\"]): \n",
    "    strat_train_set = housing.loc[train_index]\n",
    "    strat_test_set = housing.loc[test_index]\n",
    "    \n",
    "housing = strat_train_set.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16512, 11)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing = strat_train_set.drop(\"median_house_value\", axis=1)\n",
    "housing_labels = strat_train_set[\"median_house_value\"].copy()\n",
    "\n",
    "housing_num = housing.drop('ocean_proximity', axis=1)\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import TransformerMixin \n",
    "class MyLabelBinarizer(TransformerMixin):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.encoder = LabelBinarizer(*args, **kwargs)\n",
    "    def fit(self, x, y=0):\n",
    "        self.encoder.fit(x)\n",
    "        return self\n",
    "    def transform(self, x, y=0):\n",
    "        return self.encoder.transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_cat = housing[['ocean_proximity']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from future_encoders import OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "\n",
    "housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       ...,\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 1, 0]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import LabelBinarizer\n",
    "encoder = LabelBinarizer()\n",
    "housing_cat_1hot = encoder.fit_transform(housing_cat)\n",
    "housing_cat_1hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_attribs = list(housing_num)\n",
    "\n",
    "cat_attribs = [\"ocean_proximity\"]\n",
    "\n",
    "num_pipeline = Pipeline([\n",
    "        ('imputer', Imputer(strategy=\"median\")),\n",
    "        ('std_scaler', StandardScaler()),\n",
    "    ])\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "        ('selector', DataFrameSelector(cat_attribs)),\n",
    "        ('lab_encoder',  MyLabelBinarizer()),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing_num_tr = num_pipeline.fit_transform(housing_num)\n",
    "housing_cat_tr = cat_pipeline.fit_transform(housing_cat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.15604281,  0.77194962,  0.74333089, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [-1.17602483,  0.6596948 , -1.1653172 , ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [ 1.18684903, -1.34218285,  0.18664186, ...,  0.        ,\n",
       "         0.        ,  1.        ],\n",
       "       ...,\n",
       "       [ 1.58648943, -0.72478134, -1.56295222, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [ 0.78221312, -0.85106801,  0.18664186, ...,  0.        ,\n",
       "         0.        ,  0.        ],\n",
       "       [-1.43579109,  0.99645926,  1.85670895, ...,  0.        ,\n",
       "         1.        ,  0.        ]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_prepared = np.c_[housing_num_tr,housing_cat_tr]\n",
    "housing_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "forest_reg = RandomForestRegressor(random_state=42)\n",
    "forest_reg.fit(housing_prepared, housing_labels)\n",
    "housing_predictions = forest_reg.predict(housing_prepared)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53522.73342861259"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels,\n",
    "                                scoring=\"neg_mean_squared_error\", cv=3)\n",
    "forest_rmse_scores = np.sqrt(-forest_scores)\n",
    "np.mean(forest_rmse_scores)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
